package com.spider.processor;

import com.spider.entites.Filed;
import com.spider.entites.TaskSetting;
import com.spider.utils.StringUtils;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;

/**
 * 页面爬取
 * 主要定义爬取的路径和字段名称，以及主页设置
 *
 * @author panglin
 * @date 2020/11/20 21:01
 */
@Component
@Scope("prototype")
public class ListPageProcessor implements PageProcessor {

    private List<Filed> filed;

    private String next;

    private TaskSetting setting;

    public PageProcessor initPageProcessor(List<Filed> fileds,String next,TaskSetting setting){
        this.filed = fileds;
        this.next = next;
        this.setting = setting;
        return this;
    }

    @Override
    public void process(Page page) {
        this.filed.forEach(f -> {
            if (StringUtils.isNotEmpty(f.getFiled())) {
                page.putField(f.getFiled(), page.getHtml().xpath(f.getXpath()).all());
            }
        });
        if (StringUtils.isNotEmpty(this.next)) {
            page.addTargetRequest(page.getHtml().xpath(this.next).get());
        }
    }

    @Override
    public Site getSite() {
        return Site.me().setRetryTimes(setting.getRetryTime()).setSleepTime(setting.getSleepTime()).setTimeOut(setting.getTimeOut());
    }
}
